from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import time
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from random import choice
from lxml import etree
import datetime
import pymysql
from urllib import request
import re
import random
import os
import socket


# # 价格信息写入文本
# inFile = open('price.txt', 'w+')


# 用过的ip也保存下
useip=open('useip.txt','w+')


# 爬取时间段设置----------------------------------------------
# 现在的时间
now=datetime.datetime.now()
# 递增的时间
delta=datetime.timedelta(days=1)
# 30天后的时间
endnow=now+datetime.timedelta(days=30)
# 六天后的时间转换成字符串
endnow=str(endnow.strftime('%Y-%m-%d'))
now=str(now.strftime('%Y-%m-%d'))


# 全部城市路线列表-----------------------------------------------
File = open('hangxian.txt',encoding= 'utf8')
List=[]
while True:
    line=File.readline().strip()
    List.append(line)
    if len(line)==0:
        break


# 已爬取城市路线列表-----------------------------------------------
File = open('visited.txt',encoding= 'utf8')
List2=[]
while True:
    line=File.readline().strip()
    List2.append(line)
    if len(line)==0:
        break


# 链接数据库----------------------------------------------
connect = pymysql.connect(
    user="root",
    password="520134",
    port=3306,
    host="127.0.0.1",
    db="mysql",
    charset="utf8"
)
con = connect.cursor()  #获取游标
con.execute("use mysql")   #使用数据库



g_adsl_account = {"name": "taoran520yhy",
                "username": "051916738484",
                "password": "320481"}


class Adsl(object):
    def __init__(self):
        self.name = g_adsl_account["name"]
        self.username = g_adsl_account["username"]
        self.password = g_adsl_account["password"]
        self.log_file = open('log.txt', 'a')

    # set_adsl : 修改adsl设置
    def set_adsl(self, account):
        self.name = account["name"]
        self.username = account["username"]
        self.password = account["password"]

    # connect : 宽带拨号
    def connect(self):
        cmd_str = "rasdial %s %s %s" % (self.name, self.username, self.password)
        os.system(cmd_str)
        # time.sleep(5)

    # disconnect : 断开宽带连接
    def disconnect(self):
        cmd_str = "rasdial %s /disconnect" % self.name
        os.system(cmd_str)
        # time.sleep(5)

    # reconnect : 重新进行拨号
    def reconnect(self):
        self.disconnect()
        self.connect()
        try:
            name = socket.getfqdn(socket.gethostname())
            ips = socket.gethostbyname(name)
            useip.write(str(ips)+'\n')
            useip.flush()
            print('ip:'+str(ips))
        except:
            print('Can not check IP')
            useip.write('Can not check IP' + '\n')
            useip.flush()



class XiahangSpider(object):
    def __init__(self,citydate):
        self.citydate=citydate

    #模拟点击加载
    def process_request(self):
        url="https://et.xiamenair.com/xiamenair/book/findFlights.action?tripType=0&queryFlightInfo="+str(self.citydate)+","+str(now)+"&cabinClass=1&psgrInfo=0,1;1,0"
        print(url)

        # user_agent列表
        user_agent = [
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36"
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299"
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50"
        ]

        dcap = dict(DesiredCapabilities.PHANTOMJS)
        dcap["phantomjs.page.settings.loadImages"] = False  # 不载入图片，加快爬取加载速度
        dcap["phantomjs.page.settings.userAgent"] = choice(user_agent)  # 从user_agent列表中随机选一个浏览器头，伪装浏览器
        print(u'phontomJS浏览器开始模拟访问厦航官网...')
        driver=webdriver.PhantomJS(desired_capabilities=dcap)
        driver.set_page_load_timeout(30)
        driver.get(url)

        #测试是否在使用代理IP
        # driver.get("http://www.ip138.com/")
        # driver.save_screenshot("5.png")

        # 隐式等待5秒，可以自己调节
        # driver.implicitly_wait(5)

        while True:
            try:
                qqq=driver.find_elements_by_xpath("//div[@class='segment-mess']/div[@class='form-mess']/div[@class='spe-line']/a[@class='right show-more webtrends-findFlights-showMore']/span")
                for i in qqq:
                    ActionChains(driver).click(i).perform()  #对定位到的元素执行鼠标左键操作
                    print(u'正在加载当前航班所有机票价格...')
                    time.sleep(random.random()+0.8)
            except Exception as e:
                print(u'加载失败',e)

            html=driver.page_source
            response=etree.HTML(html)
            print(u'全部加载完成')

            #匹配机票信息
            for each in response.xpath("//div[@class='form-mess']"):
                hangci = each.xpath(".//span[@class='form-flt-num']/text()")[0]
                starttime = each.xpath(".//div[@class='col-lg-2 col-md-2 col-sm-2 col-xs-2']/h2[@class='bold']/text()")[0]
                endtime = each.xpath(".//div[@class='col-lg-2 col-md-2 col-sm-2 col-xs-1']/h2[@class='bold']/text()")[0]
                lists = each.xpath(".//div[@class='form-mess-inner hide-mess cabin-info spe-height lh30 clear']")
                for i in lists:
                    item1=self.citydate #出发城市，到达城市
                    dep=item1[:3]
                    arr=item1[4:]

                    flightNo= 'MF'+hangci   #航次
                    official_price= i.xpath(".//span[@class='flight-price']/text()")[0] #价格
                    cabin= i.xpath(".//span[@class='form-cabin']/text()")[0] #座位等级
                    flightTime= starttime    #出发时间
                    # item6= endtime      #到达时间
                    flightDate=response.xpath("//div[@class='nav-outer low-price-nav']//li[@class='this-tab']//h3/@date")[0] #出发日期
                    uptime=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) #爬取时间
                    # print(dep,arr,flightNo,official_price,cabin,flightTime,flightDate,uptime)
                    # inFile.write(
                    #     dep + ',' + arr + ',' + flightNo + ',' + official_price + ',' + cabin + ',' + flightTime + ',' + flightDate + ',' + uptime + '\n')
                    # inFile.flush()

                    try:
                        con.execute("replace into ceshi(dep,arr,flightNo,official_price,cabin,flightTime,flightDate,uptime)values"
                                    "(%s,%s,%s,%s,%s,%s,%s,%s)",[dep, arr, flightNo, official_price, cabin, flightTime, flightDate, uptime])
                        connect.commit()  # 我们需要提交数据库，否则数据还是不能上传的
                        print('数据正在上传...')
                    except Exception as e:
                        print('数据上传失败...',e)
                        pass

            # 定位下一日期
            date=response.xpath('//*[@id="load-mess"]/div[1]/div[1]/div[2]/div[1]/ul/li[5]/a/h3/@date')[0]
            # print(date)
            # print(endnow)

            # 当下一日期等于指定日期，就停止循环
            if str(date) == endnow:
                driver.quit()
                break

            # 随机切换ip策略
            number=random.randint(1,5)
            if number==1:
                try:
                    print(u'即将重新拨号切换ip')
                    adsl = Adsl()
                    adsl.reconnect()
                except:
                    print(u'切换失败,尝试重连')
                    adsl = Adsl()
                    adsl.connect()

            # 一直点击下一页
            qq=driver.find_element_by_xpath('//*[@id="load-mess"]/div[1]/div[1]/div[2]/div[1]/ul/li[5]')
            ActionChains(driver).click(qq).perform()
            time.sleep(0.8+random.random())




#判断航线是否开通
def judgeURL(city):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36"
    }
    url = "https://et.xiamenair.com/xiamenair/book/findFlights.action?tripType=0&queryFlightInfo=" + str(city) + "," + str(now) + "&cabinClass=1&psgrInfo=0,1;1,0"
    # print(url + ' ' + city)
    req = request.Request(url, headers=headers)
    html = request.urlopen(req).read()

    page = html.decode('utf-8')
    m = re.search("<title>.*</title>", page)
    if m == None:
        print("好棒，这条线路开通啦")
        return True
    else:
        print("这条线路未开通",city)
        return False


if __name__ == "__main__":
    # lists = ['CGO,DLC', 'CGO,XXX', 'CGO,HET', 'CGO,JJN', 'CGO,DNH', 'BFJ,XMN', 'CGO,HRB', 'CGO,INC', 'CAN,TSN',
    #          'CSO,XOC', 'BAV,HHA', 'CGO,HGH', 'CAN,HGH', 'CGO,CKG', 'CGO,HAK', 'BAV,CSX', 'CAN,XMN', 'CGO,FOC',
    #          'CGO,HLD', 'CGO,CGQ', 'CAN,FOC', 'CAN,JJN']

    # 打开visited.txt，将爬取过的路线写入
    visited = open('visited.txt', 'a+')

    for i in List:
        if i in List2:
            print('%s航线已经爬取过了'%(i))
        else:
            print("正在判断%s航线是否开通"%(i))
            if judgeURL(i):
                # 若航线开通则进行抓取
                a = XiahangSpider(i)
                a.process_request()
                time.sleep(3 + random.random())
            else:
                pass

            # 爬取过的路线 写入到专门的文本
            visited.write(str(i) + '\n')
            visited.flush()
    con.close()  # 关闭游标
    connect.close()  # 关闭数据库


















